Preliminaries

Load packages. If you don’t have them installed already, you should uncomment (delete the #) the first line in the following code block (starting ‘install.packages’)

Once you’ve done that, you can test the file by using the ‘knit’ button (in RStudio).

Read more about R markdown and ‘kniting’ (rendering) documents https://rmarkdown.rstudio.com/authoring_quick_tour.html#overview

To read documentation and view examples of usage, type ?function_name in console (E.g. ?hist) or search from the help bar in RStudio.

knitr::opts_chunk$set(warning = FALSE)
#install.packages(c("psych","ggplot2","doBy","reshape2","knitr","lattice"))
  sh <- suppressPackageStartupMessages #To get rid os warning and other messages while loading the libraries
  sh(library(ggplot2))  #for graphs and plots
  sh(library(psych))    #for statistical measures and testing
  #sh(library(doBy))     #for group by analysis dplyr covers this
  #sh(library(reshape2)) #for data wrangling
  sh(library(knitr))    #for rendering markdown
  #sh(library(lattice))  #just to illustrate another histogram function 
  library(learnr)
  library(dslabs)
  library(shiny) #shouldn't be necessary but...
 # install.packages("remotes")
  #remotes::install_github("rstudio-education/gradethis")
  library(gradethis)
  gradethis_setup()
  library(textdata)
  library(wordcloud) #ggwordcloud could also be used to create comparison.cloud but for now I'll keep 
  library(ggwordcloud) #This library is interesting, it does the same things as wordcloud + wordcloud2 combined, and is in the ggplot2 family which means we can do all those things too
  #library(wordcloud2) #This may not work well when knitted
  library(kableExtra)
  #library(RSentiment) #for ease, takes whole sentences and assigns scores in various ways (while tidytext approaches use words) Described e.g. https://www.r-bloggers.com/2017/03/rsentiment/ Imports OpenNLP and NLP in addition to ^. We'd use function calculate_score 
  library(sentimentr) #more complete than RSentiment, many examples. Compares a range of approaches https://cran.r-project.org/web/packages/sentimentr/readme/README.html  Also has a nice function plot.sentiment_by which gives a ggplot object
  #this is great! https://github.com/trinker/sentimentr#examples
  library(tidyverse)
  library(tidytext)
  library(corrplot)
#detach(package:plyr)
  
#library(rsconnect)
#deployApp(appName = "Snippet_matchR)
#################################################################
#############LOAD DATA HERE######################################
#################################################################
#trump_tweets <- data("trump_tweets")
data("trump_tweets")
##############AND we're going to do some tidying up #############

links <- "https://t.co/[A-Za-z\\d]+|&amp;" #regex to get rid of picture links
tweet_words <- trump_tweets %>% 
  mutate(text = str_replace_all(text, links, ""))  %>%
  unnest_tokens(word, text, token = "tweets") %>%
  filter(!word %in% stop_words$word &
           !str_detect(word, "^\\d+$")) %>%
  mutate(word = str_replace(word, "^'", ""))

## Using `to_lower = TRUE` with `token = 'tweets'` may not preserve URLs.

##############And add sentiment analysis columns for later #########
bing <- get_sentiments("bing")
afinn <- get_sentiments("afinn")

#loughran <- get_sentiments("loughran") %>% count(sentiment)
#get_sentiments("nrc") %>% count(sentiment) 

nrc <- get_sentiments("nrc") %>%
  select(word, sentiment)

#Find the sentiment for each word, then count each sentiment for each tweet (using id_str as index), add these as columns using pivot_wider, and subtract from each other to give an overall score.  This isn't really correct, the words should be weighted by n of words, to give a score 0-1, or -1-+1 but it's fine for now.

trump_tweets %<>% 
  dplyr::mutate(sent_split = get_sentences(text)) %>%
  dplyr::mutate(sentiment_by(sent_split)) %>%
  dplyr::mutate(polarity = ifelse(ave_sentiment < 0.2, "Negative",
                           ifelse(ave_sentiment > 0.2, "Positive","Neutral")))


tt_senti2 <- tweet_words %>%
  inner_join(get_sentiments("afinn")) %>%
  group_by(id_str) %>%
  dplyr::summarise(sentiment_afinn = mean(value)) 

## Joining, by = "word"

#mode <- function(codes){which.max(tabulate(codes))}
mode <- function(x) { names(which.max(table(x))) } #it's treated as factor below

tt_senti3 <- tweet_words %>%
  inner_join(get_sentiments("nrc")) %>%
  group_by(id_str) %>%
  dplyr::summarise(sentiment_nrc = mode(sentiment))

## Joining, by = "word"

#join the sentiment to the original twitter data
#trump_tweets <- left_join(trump_tweets, tt_senti, by = c("id_str" = "index")) 
trump_tweets <- left_join(trump_tweets, tt_senti2, by = "id_str") 
trump_tweets <- left_join(trump_tweets, tt_senti3, by = "id_str") 

rm(tt_senti2,tt_senti3)

#table(tweet_words %>% inner_join(get_sentiments("nrc")) %>% select(sentiment)) 
#afinn is also pretty cool

Data Exploration

First, let’s just look at the first 10 rows of the dataset to see what it includes. You can change the below to tail(trump_tweets) or trump_tweets[1:100,], etc. Use the arrow in the top row to navigate through the columns.

#View the first 10 rows of the dataset
head(trump_tweets)

Demonstrating a Table / Data Overview

Let’s just take a look at some overall summary stats before we get into the visualisation. These are useful for a range of reasons, one of which is they can tell us about the shape of the data. Read about a part of that through measures of skew and kurtosis here: https://codeburst.io/2-important-statistics-terms-you-need-to-know-in-data-science-skewness-and-kurtosis-388fef94eeaa

trump_tweets %>% 
  filter(source %in% c("Twitter for Android","Twitter for iPhone","TweetDeck","Twitter Web Client")) %>% 
  select(c(favorite_count,retweet_count,source)) %>%
  psych::describeBy(favorite_count + retweet_count ~ source, data = ., mat = T) %>%
  kable()

Some demonstration

An example (a pretty ugly one, using base R) of a plot is in the code. We will see how this can be improved later. One easy fix we’ll apply later is to rotate the x axis to show the number, not the notation. Unfortunately base R doesn’t let us do that easily.

hist(trump_tweets$favorite_count)

We’re also going to quickly demonstrate how part of this page works. The page is built on an r package called learnr which lets me insert exercises that you can run yourselves, and lets me check your answers! Cool right?

Here’s a really basic example. In the box below, I want you to assign the value 2 to the variable x. In r we assign by using the operator <- This is preceded by the name of the variable, in this case x And followed by the thing we’re assigning, in this case 2 So y <- 3*3 assigns the value 9 to the variable y Delete the code in the box to do this; the checker looks at your code rather than the output in this case.

1+1

## [1] 2

x <- 2

grade_code()

## function (check_env) 
## {
##     if (is.list(check_env)) {
##         check_env <- list2env(check_env)
##     }
##     user_code <- check_env$.user_code
##     if (is.null(user_code)) {
##         return(legacy_graded(correct = FALSE, message = "I didn't receive your code. Did you write any?"))
##     }
##     solution_code <- check_env$.solution_code
##     if (is.null(solution_code) || length(str2expression(solution_code)) == 
##         0) {
##         return(legacy_graded(correct = FALSE, message = "No exercise solution provided. Defaulting to _incorrect_."))
##     }
##     message <- code_feedback(user_code = user_code, solution_code = solution_code, 
##         env = check_env, allow_partial_matching = allow_partial_matching)
##     if (is.null(message)) {
##         return(legacy_graded(correct = TRUE, message = glue_message(glue_correct %||% 
##             gradethis_legacy_options$gradethis.glue_correct, 
##             .is_correct = TRUE, .message = NULL, .correct = correct, 
##             .user_code = user_code)))
##     }
##     message <- glue_message(glue_incorrect %||% gradethis_legacy_options$gradethis.glue_incorrect, 
##         .is_correct = FALSE, .message = message, .incorrect = incorrect, 
##         .user_code = user_code)
##     legacy_graded(correct = FALSE, message = message)
## }
## <bytecode: 0x0000022af7c543b8>
## <environment: 0x0000022a8bfa6a88>

We’ve made it easy below. You don’t need to write your own code, just copy and paste our examples into the correct slot!

Q1. What is the relationship between retweets and likes?

ggplot(trump_tweets, aes(x = retweet_count, y = favorite_count)) +
  geom_point(shape = 1) 

cor.test(trump_tweets$retweet_count,trump_tweets$favorite_count)

## 
##  Pearson's product-moment correlation
## 
## data:  trump_tweets$retweet_count and trump_tweets$favorite_count
## t = 331.95, df = 20759, p-value < 2.2e-16
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
##  0.9151331 0.9194466
## sample estimates:
##       cor 
## 0.9173168

grade_code()

## function (check_env) 
## {
##     if (is.list(check_env)) {
##         check_env <- list2env(check_env)
##     }
##     user_code <- check_env$.user_code
##     if (is.null(user_code)) {
##         return(legacy_graded(correct = FALSE, message = "I didn't receive your code. Did you write any?"))
##     }
##     solution_code <- check_env$.solution_code
##     if (is.null(solution_code) || length(str2expression(solution_code)) == 
##         0) {
##         return(legacy_graded(correct = FALSE, message = "No exercise solution provided. Defaulting to _incorrect_."))
##     }
##     message <- code_feedback(user_code = user_code, solution_code = solution_code, 
##         env = check_env, allow_partial_matching = allow_partial_matching)
##     if (is.null(message)) {
##         return(legacy_graded(correct = TRUE, message = glue_message(glue_correct %||% 
##             gradethis_legacy_options$gradethis.glue_correct, 
##             .is_correct = TRUE, .message = NULL, .correct = correct, 
##             .user_code = user_code)))
##     }
##     message <- glue_message(glue_incorrect %||% gradethis_legacy_options$gradethis.glue_incorrect, 
##         .is_correct = FALSE, .message = message, .incorrect = incorrect, 
##         .user_code = user_code)
##     legacy_graded(correct = FALSE, message = message)
## }
## <bytecode: 0x0000022af7c543b8>
## <environment: 0x0000022a8525d440>

#block 8 - correct
#block 7 and 17
## #Is date an important variable in this analysis? Does the scaling of the data gives us the best available insight into relationships of paired values? 

Q2. What source has the highest sentiment?

For this one, we can see four tweets in the same range, what are the values and their content (the tweet text)?

trump_tweets %>% 
  filter(source %in% c("Twitter for Android","Twitter for iPhone")) %>% #filter to the two biggest sources
  ggplot(aes(x = factor(source), y = ave_sentiment)) +  #note, you'll often see , fill = factor(variable) here too, but unless it is adding additional information my view is colour is just visual noise, the axes already label the plots, adding colour doesn't add new information.
  geom_boxplot() +
  stat_summary(
    fun.y = mean, geom="point", shape=5, size=4)

trump_tweets %>% 
  filter(source == "Twitter for Android") %>%
  slice_max(ave_sentiment, n = 1) %>%
  select(text,retweet_count,favorite_count,created_at,source,ave_sentiment) #uses dplyr slice function, and selects the columns we're interested in

trump_tweets %>% 
  filter(source == "Twitter for iPhone") %>%
  slice_max(ave_sentiment, n = 1) %>%
  select(text,retweet_count,favorite_count,created_at,source,ave_sentiment) #uses dplyr slice function, and selects the columns we're interested in

grade_code()

## function (check_env) 
## {
##     if (is.list(check_env)) {
##         check_env <- list2env(check_env)
##     }
##     user_code <- check_env$.user_code
##     if (is.null(user_code)) {
##         return(legacy_graded(correct = FALSE, message = "I didn't receive your code. Did you write any?"))
##     }
##     solution_code <- check_env$.solution_code
##     if (is.null(solution_code) || length(str2expression(solution_code)) == 
##         0) {
##         return(legacy_graded(correct = FALSE, message = "No exercise solution provided. Defaulting to _incorrect_."))
##     }
##     message <- code_feedback(user_code = user_code, solution_code = solution_code, 
##         env = check_env, allow_partial_matching = allow_partial_matching)
##     if (is.null(message)) {
##         return(legacy_graded(correct = TRUE, message = glue_message(glue_correct %||% 
##             gradethis_legacy_options$gradethis.glue_correct, 
##             .is_correct = TRUE, .message = NULL, .correct = correct, 
##             .user_code = user_code)))
##     }
##     message <- glue_message(glue_incorrect %||% gradethis_legacy_options$gradethis.glue_incorrect, 
##         .is_correct = FALSE, .message = message, .incorrect = incorrect, 
##         .user_code = user_code)
##     legacy_graded(correct = FALSE, message = message)
## }
## <bytecode: 0x0000022af7c543b8>
## <environment: 0x0000022af5a25460>

#block 13
#Distractor is block 2
#For interest, you could also look at the lowest sentiment two
# trump_tweets %>% 
#   filter(source == "Twitter for Android") %>%
#   slice_min(ave_sentiment, n = 1) %>%
#   select(text,retweet_count,favorite_count,created_at,source,ave_sentiment) #uses dplyr slice function, and selects the columns we're interested in
# 
# trump_tweets %>% 
#   filter(source == "Twitter for iPhone") %>%
#   slice_min(ave_sentiment, n = 1) %>%
#   select(text,retweet_count,favorite_count,created_at,source,ave_sentiment) #uses dplyr slice function, and selects the columns we're interested in

Q3. How do iphone and android compare in terms of number of words?

#Second version - sometimes it's useful to think about what information is included in different representations
trump_tweets %>% 
  filter(source %in% c("Twitter for Android","Twitter for iPhone","TweetDeck","Twitter Web Client")) %>% #filter to the  biggest sources
  ggplot(aes(x = factor(source), y = word_count)) +  #note, you'll often see , fill = factor(variable) here too, but unless it is adding additional information my view is colour is just visual noise, the axes already label the plots, adding colour doesn't add new information.
  geom_boxplot() +
  stat_summary(
    fun.y = mean, geom="point", shape=5, size=4)

kable(trump_tweets$source)